# -*- coding:utf-8 -*-
from pymongo import MongoClient
from config import config
from bs4 import BeautifulSoup
import re

# 该脚本的作用是从网页中提取感兴趣的数据


def do_it_one(source_page):
    print source_page['url']
    result = {
        "url": source_page['url'],
        "title": '',
        "vote_num": 0,
        "vote_value": -1
    }
    content = source_page['content']
    bs_obj = BeautifulSoup(content, 'lxml')

    # 标题
    title = bs_obj.find("span", property="v:itemreviewed")
    if title is not None:
        result['title'] = title.text

    # 评价人数
    rate_num = bs_obj.find("span", property="v:votes")
    if rate_num is not None:
        result['vote_num'] = int(rate_num.text)

    # 评价得分
    rate_value = bs_obj.find("strong", property="v:average")
    # print rate_value
    if rate_value is not None:
        rate_value = rate_value.text.replace(" ","")
        if len(rate_value)>0:
            result['vote_value'] = float(rate_value) 

    # 书籍信息
    info = bs_obj.find("div", id="info")
    book_info = []
    if info is not None:
        temp1 = str(info).replace("\n","").split("<br/>")
        # print temp1
        temp2 = []
        temp3 = []
        count = 0
        for item in info.findAll("span",{"class":"pl"}):
            count += 1
            temp2.append(str(item))
            temp3.append(item.text.replace(" ","").replace(":",""))
        # print temp2
        for i in range(0, count):
            # print temp1[i]
            # print temp2[i]
            # print temp3[i]
            temp4 = temp1[i].replace(temp2[i],"").replace(":","")
            temp4_bs_obj = BeautifulSoup(temp4, 'lxml')
            temp4 = temp4_bs_obj.html
            if temp4 is None:
                book_info.append({"key": temp3[i], "value": ""})
            else:
                temp4 = temp4.text.replace(" ", "")
                # print temp3[i]
                if temp3[i] == u'作者':
                    # exit()
                    if temp4.find('[') == 0 or temp4.find('(') == 0 or temp4.find(u'（') == 0 or temp4.find(u'【') == 0 or temp4.find(u'〔') == 0:
                        temp5 = re.findall(ur'[\[\(（【〔［]([^\]^\[^（^）^/^【^】^〔^〕^［^］]*)[\]\)）】〕］]'
                                           ur'([^\]^\[^（^）^/^【^】^〔^〕^［^］]*)', temp4)
                        if len(temp5) >= 1 and len(temp5[0]) == 2:
                            book_info.append({"key": "作者", "value": {
                                "name": temp5[0][1], "country": temp5[0][0]
                            }})
                        else:
                            book_info.append({"key": temp3[i], "value": {"name": temp4}})
                    else:
                        book_info.append({"key": temp3[i], "value": {"name": temp4}})
                else:
                    book_info.append({"key": temp3[i], "value": temp4})
    result['info'] = book_info

    # 书籍简介
    temp = bs_obj.find("div", id="link-report")
    if temp is not None:
        temp = temp.findAll("div",{"class":"intro"})
        if temp is not None:
            result['book_introduce'] = str(temp[len(temp)-1])
    # print result['book_introduce']

    # tag列表
    tags = []
    temp = bs_obj.find("div",id="db-tags-section")
    if temp is not None:
        for item in temp.findAll("a",{"class":"tag"}):
            tags.append(item.text.replace(" ",""))
        result['tags'] = tags

    return result


def main():
    mongodb_client = MongoClient(config.mongodb_host, config.mongodb_port)
    mongodb_db = mongodb_client[config.mongodb_db]
    mongodb_table_source_page = mongodb_db[config.mongodb_table_source_page]

    mongodb_table_book_info = mongodb_db[config.mongodb_table_book_info]

    limit = 1000
    offset = 0
    while True:
        print offset
        items = mongodb_table_source_page.find({
            'url': {
                '$regex': '^/subject/\d+/$'
            }
        }).limit(limit).skip(offset)
        count = 0
        for item in items:
            data = do_it_one(item)
        # print data
            # print data #['url'], '#', data['title'], '#', data['vote_num']
            # exit()
            print mongodb_table_book_info.insert_one(data).inserted_id
        # exit()
            count += 1
        if count == 0:
            break
        offset += limit

if __name__ == '__main__':
    main()
